Reproducible Documentation of Analysis Study 1

Import of the Data

The data was assessed with the formr survey framework (Arslan, Walther, and Tata 2020). The raw data was imported via the following code.

library(tidyverse)

── Attaching packages ─────────────────────────────────────── tidyverse 1.3.2 ──
✔ ggplot2 3.4.0      ✔ purrr   0.3.5 
✔ tibble  3.1.8      ✔ dplyr   1.0.10
✔ tidyr   1.2.1      ✔ stringr 1.4.1 
✔ readr   2.1.3      ✔ forcats 0.5.2 
── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
✖ dplyr::filter() masks stats::filter()
✖ dplyr::lag()    masks stats::lag()

library(ggforce)
library(lme4)

Loading required package: Matrix

Attaching package: 'Matrix'

The following objects are masked from 'package:tidyr':

    expand, pack, unpack

library(BFpack)

Loading required package: bain
Registered S3 methods overwritten by 'BFpack':
  method               from
  get_estimates.lm     bain
  get_estimates.t_test bain

library(hrbrthemes)
library(patchwork)
library(brms)

Loading required package: Rcpp
Loading 'brms' package (version 2.18.0). Useful instructions
can be found by typing help('brms'). A more detailed introduction
to the package is available through vignette('brms_overview').

Attaching package: 'brms'

The following object is masked from 'package:lme4':

    ngrps

The following object is masked from 'package:stats':

    ar

library(viridis)

Loading required package: viridisLite

library(ggdist)

Attaching package: 'ggdist'

The following objects are masked from 'package:brms':

    dstudent_t, pstudent_t, qstudent_t, rstudent_t

library(tidybayes)

Attaching package: 'tidybayes'

The following objects are masked from 'package:brms':

    dstudent_t, pstudent_t, qstudent_t, rstudent_t

load("../data/teachers_study1_N40.RData")

set.seed(25051982)

Data Wrangling

# wrangle information on the plot type, ES, ...
plot_info <- study1 %>%
    pivot_longer(2:195, names_to = "variables", values_to = "values", 
                 values_transform = as.character) %>%
    dplyr::filter(str_detect(variables, "plot")) %>% 
    # we only need the rows with info on plots
    tidyr::separate(col = values, into = c("type", "axis", "effsize"), 
                    # separate the info into three columns
                    sep = "_", remove = F) %>%
    dplyr::mutate(plot = variables,       # rename variables for later join
                  type = paste(type, axis, sep = "_")) %>%
    dplyr::select(-variables, -axis)

# wrangle answers to items on each page
item_values <- study1 %>%
    dplyr::select(-c(topic:itemo)) %>%
    pivot_longer(2:169, names_to = "variables", values_to = "values", 
                 values_transform = as.character) %>%
    dplyr::mutate(variables = case_when(      # recode variable names that have
        variables == "sensi_6" ~ "sensi_06",  # accidentally been labeled
        variables == "acccl_6" ~ "acccl_06",  # without zero
        variables == "accu3_6" ~ "accu3_06",
        variables == "accov_6" ~ "accov_06",
        variables == "diffi_6" ~ "diffi_06",
        variables == "infor_6" ~ "infor_06",
        variables == "value_6" ~ "value_06",
        TRUE ~ variables 
    )) %>%
    dplyr::mutate(plot = paste0("plotx_", str_sub(variables, -2, -1)), 
                  # create variable for later join
                  variables = str_sub(variables, 1, -4)) %>%    
    # rename variable names to get a data set 
    # with one line per participant per page
    pivot_wider(id_cols = c(session, plot), names_from = "variables", 
                values_from = "values")


# join the two data sets
study1_w <- full_join(plot_info, item_values, 
                               by = c("session", "plot")) %>% 
    # by participant and page (plot)
    dplyr::select(-values) %>%
    dplyr::mutate(rating_cl = as.numeric(acccl), # some var need to be defined as
                  rating_u3 = as.numeric(accu3), # numeric again
                  rating_ov = as.numeric(accov),
                  diffi = as.numeric(diffi),
                  infor = as.numeric(infor),
                  value = as.numeric(value),
                  effsize = as.numeric(effsize),
                  effsize_cl = case_when( 
                  # there is no negative Cliff's Delta, so we have to compute 
                  # two transformations
                      effsize > 0 ~   (((2*pnorm(effsize/2))-1)/pnorm(effsize/2)),
                  # transform the actual effect size Cohen's d to Cliff's Delta
                      effsize < 0 ~ (- (((2*pnorm(abs(effsize)/2))-1)/pnorm(abs(effsize)/2))) 
                  # transform the actual effect size Cohen's d to Cliff's Delta 
                  # and make it negative as in the item
                  ),
                  effsize_u3 = 1-pnorm(effsize), # reverse so that it fits the direction of the U3 item
                  # transform the actual effect size Cohen's d to Cohen's U3
                  effsize_ov = 2 * pnorm(-abs(effsize) / 2), 
                  # transform the actual effect size Cohen's d to overlap
                  # actual difference of rating relative to depicted effectsize 
                  diff_cl = (rating_cl - effsize_cl)/2,
                  # actual difference of rating relative to depicted effectsize
                  diff_u3 = (rating_u3/100) - effsize_u3,
                  # actual difference of rating relative to depicted effectsize 
                  diff_ov = (rating_ov/100) - effsize_ov,
                  diffi_normed = ((diffi - 1)  / 3) - 1, # transform item to -1 to 1
                  infor_normed = ((infor - 1)  / 3) - 1, # transform item to -1 to 1
                  value_normed = ((value - 1)  / 3) - 1) %>%  # transform item to -1 to 1
    group_by(session) %>% 
    mutate(rating_ov_missconcept = median(rating_ov, na.rm = T) < 68.9,
           rating_u3_missconcept = median(rating_u3, na.rm = T) < 21.2) %>% 
    ungroup() %>% 
    mutate(rating_u3_filtered = ifelse(rating_u3_missconcept == T, NA, rating_u3),
           rating_ov_filtered = ifelse(rating_ov_missconcept == T, NA, rating_ov),
           diff_u3_filtered = (rating_u3_filtered/100) - effsize_u3,
           diff_ov_filtered = (rating_ov_filtered/100) - effsize_ov,
           sensi_binary = ifelse(is.na(sensi), # 1 if NOT "equal"
                                        NA,
                                        as.numeric(!grepl("equal", sensi))),
           sensi_ordinal = ordered(factor(substr(sensi, 55, 100)),
                                   levels = c("inferior",
                                              "equal",
                                              "superior")),
           sensi_binary_filtered = case_when(sensi_ordinal == "equal" ~ 0, 
                                              (sensi_ordinal == "inferior" & 
                                                  effsize < 0) | 
                                                  (sensi_ordinal == "superior" & 
                                                  effsize > 0) ~ as.numeric(NA),
                                              TRUE ~ 1),
           effsize_abs = abs(effsize))

# create a list of u3_misconceptualizers
u3_misconceptualizers <-
    study1_w %>% 
    filter(rating_u3_missconcept == T) %>% 
    pull(session) %>% 
    unique()

# create a list of ov_misconceptualizers
ov_misconceptualizers <-
    study1_w %>% 
    filter(rating_ov_missconcept == T) %>% 
    pull(session) %>% 
    unique()    

### wrangle time stamp data ####################################################
study1_w_timestamp <- 
    read_csv("../data/teachers_study1_N40_detailed.csv") %>% 
    # filter participants from study1_w only
    filter(session %in% study1_w$session) %>% 
    # we only need vars sensitivity or accuracy
    dplyr::filter(str_detect(item_name, "sensi|acccl|accu3|accov")) %>%  
    # create var with plot number
    mutate(plot = paste0("plotx_", str_sub(item_name, -2, -1)),
           # recode wrong item labelling
           plot = ifelse(plot == "plotx__6", "plotx_06", plot)) %>% 
    relocate(session, plot) %>% 
    # delete the page number in item name
    mutate(item_name = str_sub(item_name, 1, 5)) %>%  
    pivot_wider(id_cols = c(session, plot), names_from = item_name, 
                values_from = answered_relative) %>% 
    rowwise() %>%
    # what was the time of the first item to be clicked?
    mutate(effic = min(sensi, acccl, accu3, accov, na.rm=T)) %>%
    ungroup() %>% 
    dplyr::select(session, plot, effic, sensi, acccl, accu3, accov) %>% 
    left_join(., study1_w %>% 
                  select(session, plot, type), by=c("session", "plot")) %>% 
    # generate data set so that the six plots from the same type are ordered
    # one after the other (and not 1-24)
    group_by(session, type) %>% 
    arrange(plot) %>% 
    mutate(plotNrWithin = 1:n()) %>%
    ungroup() %>% 
    group_by(plotNrWithin, type) %>% 
    mutate(effic_10righttrunc = ifelse(effic > quantile(effic, .9), NA, effic),
           effic_05righttrunc = ifelse(effic > quantile(effic, .95), NA, effic),
           log_effic_05righttrunc =log(effic_05righttrunc),
           log_effic_10righttrunc = log(effic_10righttrunc),
           plotNrWithin0 = plotNrWithin -1,
           plotNrWithin_factor = as.factor(plotNrWithin)) %>% 
    ungroup()

Rows: 27336 Columns: 14
── Column specification ────────────────────────────────────────────────────────
Delimiter: ","
chr  (3): session, item_name, answer
dbl  (7): unit_session_id, item_id, shown_relative, answered_relative, displ...
dttm (4): created, saved, shown, answered

ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.

Accuracy

Distribution of the Dependent Variables

study1_w %>% 
    select(rating_cl, rating_u3, rating_ov) %>% 
    pivot_longer(
        c(rating_cl, rating_u3, rating_ov),
        names_to = "dependent_variable", 
        values_to = "rated_effectsize"
        ) %>% 
    ggplot(., aes(rated_effectsize)) +
    geom_histogram() +
    facet_wrap(~dependent_variable, scales = "free_x") +
    theme_modern_rc() +
    theme(strip.text = element_text(color = "white"))

`stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

Warning: Removed 1440 rows containing non-finite values (`stat_bin()`).

Somewhat disturbing is the first mode in rating_ov. Maybe some users confused overlap and non-overlap? Another artifact seems to be the first mode in rating_u3.

Are there Constant Misconceptions per Persons?

ggplot(study1_w %>% 
    select(rating_cl, rating_u3, rating_ov, effsize, effsize_cl, session) %>% 
    pivot_longer(
        c(rating_cl, rating_u3, rating_ov),
        names_to = "operationalization", 
        values_to = "judged_effectsize"
        ),
    aes(judged_effectsize, as.numeric(as.factor(session)),
        color = session)
        ) +
    geom_jitter(height = 0) +
    facet_wrap(~ operationalization, scales = "free_x") +
    theme_modern_rc() +
    theme(legend.position = "none",
          strip.text = element_text(color = "white"))

Warning: Removed 1440 rows containing missing values (`geom_point()`).

Association of Ratings and Actual Effect Size

study1_w %>% 
    ggplot(., aes(effsize, rating_cl)) +
    geom_jitter() +
    stat_smooth() +
    theme_modern_rc()

`geom_smooth()` using method = 'loess' and formula = 'y ~ x'

Warning: Removed 480 rows containing non-finite values (`stat_smooth()`).

Warning: Removed 480 rows containing missing values (`geom_point()`).

study1_w %>% 
    ggplot(., aes(effsize, rating_u3, color = rating_u3_missconcept)) +
    geom_jitter() +
    stat_smooth() +
    theme_modern_rc()

`geom_smooth()` using method = 'loess' and formula = 'y ~ x'

Warning: Removed 480 rows containing non-finite values (`stat_smooth()`).
Removed 480 rows containing missing values (`geom_point()`).

study1_w %>% 
    ggplot(., aes(abs(effsize), rating_ov, color = rating_ov_missconcept)) +
    geom_jitter() +
    stat_smooth(method = "lm") +
    theme_modern_rc()

`geom_smooth()` using formula = 'y ~ x'

Warning: Removed 480 rows containing non-finite values (`stat_smooth()`).
Removed 480 rows containing missing values (`geom_point()`).

The correlations underpin the interpretation of the misconceptions. Therefore, we will look at the intercorrelations of the dependent variables.

Compute Kendall’s \(\tau\) for rating_cl, rating_u3 and rating_ov

study1_w %>% 
    group_by(type) %>% 
    do(tau_cl = unlist(cor(.$effsize, .$rating_cl, method = "kendall", 
                           use = "pairwise.complete"))) %>% 
    unnest(tau_cl)

# A tibble: 4 × 2
  type                tau_cl
  <chr>                <dbl>
1 gardneraltman_xaxis  0.373
2 halfeye_xaxis        0.548
3 halfeye_yaxis        0.484
4 raincloud_yaxis      0.415

study1_w %>% 
    group_by(rating_ov_missconcept, type) %>% 
    do(tau_ov = unlist(cor(abs(.$effsize), .$rating_ov, method = "kendall", 
                           use = "pairwise.complete"))) %>% 
    unnest(tau_ov)

# A tibble: 8 × 3
  rating_ov_missconcept type                 tau_ov
  <lgl>                 <chr>                 <dbl>
1 FALSE                 gardneraltman_xaxis -0.355 
2 FALSE                 halfeye_xaxis       -0.384 
3 FALSE                 halfeye_yaxis       -0.478 
4 FALSE                 raincloud_yaxis     -0.523 
5 TRUE                  gardneraltman_xaxis  0.0236
6 TRUE                  halfeye_xaxis        0.119 
7 TRUE                  halfeye_yaxis        0.0205
8 TRUE                  raincloud_yaxis     -0.220 

study1_w %>% 
    group_by(rating_u3_missconcept, type) %>% 
    do(tau_u3 = cor(.$effsize, .$rating_u3, method = "kendall", 
                           use = "pairwise.complete")) %>% 
    unnest(tau_u3)

# A tibble: 8 × 3
  rating_u3_missconcept type                 tau_u3
  <lgl>                 <chr>                 <dbl>
1 FALSE                 gardneraltman_xaxis -0.298 
2 FALSE                 halfeye_xaxis       -0.471 
3 FALSE                 halfeye_yaxis       -0.263 
4 FALSE                 raincloud_yaxis     -0.321 
5 TRUE                  gardneraltman_xaxis  0.119 
6 TRUE                  halfeye_xaxis       -0.153 
7 TRUE                  halfeye_yaxis       -0.108 
8 TRUE                  raincloud_yaxis     -0.0971

Associations of the Dependent Variables

ggplot(study1_w, 
       aes(rating_cl, rating_u3)) +
    geom_jitter() +
    theme_modern_rc() +
ggplot(study1_w, 
       aes(rating_cl, rating_ov)) +
    geom_jitter() +
    theme_modern_rc() +
ggplot(study1_w, 
       aes(rating_ov, rating_u3)) +
    geom_jitter() +
    theme_modern_rc()

Warning: Removed 480 rows containing missing values (`geom_point()`).
Removed 480 rows containing missing values (`geom_point()`).
Removed 480 rows containing missing values (`geom_point()`).

Global Under- or Overestimation

study1_w %>% 
    select(diff_cl, diff_u3_filtered, diff_ov_filtered, session, type) %>% 
    gather(dependent_variable, difference_to_true_effsize, 
           diff_cl, diff_u3_filtered, diff_ov_filtered) %>% 
    ggplot(., aes(type, difference_to_true_effsize)) +
    geom_boxplot(alpha = .3) +
    facet_wrap(~dependent_variable, scales = "free_y") +
    geom_jitter(aes(color = type)) +
    theme_modern_rc() +
    theme(strip.text = element_text(color = "white"),
          axis.text.x = element_blank())

Warning: Removed 1812 rows containing non-finite values (`stat_boxplot()`).

Warning: Removed 1812 rows containing missing values (`geom_point()`).

Research Question 1

Intraclasscorrelations for Accuracy, Perceived Difficulty, Informativity and Value

icc_mod_rating_cl <- lmer(rating_cl ~  1 + (1|type), 
                          data = study1_w )

boundary (singular) fit: see help('isSingular')

icc_mod_rating_u3 <- lmer(rating_u3 ~  1 + (1|type), 
                          data = study1_w %>% 
                              filter(rating_u3_missconcept == F))
icc_mod_rating_ov <- lmer(rating_ov ~  1 + (1|type), 
                          data = study1_w %>% 
                              filter(rating_ov_missconcept == F))
icc_mod_diffi <- lmer(diffi ~  1 + (1|type), 
                      data = study1_w)
icc_mod_infor <- lmer(infor ~  1 + (1|type), 
                      data = study1_w )
icc_mod_value <- lmer(value ~  1 + (1|type), 
                      data = study1_w )

sjPlot::tab_model(
    icc_mod_rating_cl,
    icc_mod_rating_u3,
    icc_mod_rating_ov,
    icc_mod_diffi,
    icc_mod_infor,
    icc_mod_value
)

	rating_cl			rating_u3			rating_ov			diffi			infor			value
Predictors	Estimates	CI	p	Estimates	CI	p	Estimates	CI	p	Estimates	CI	p	Estimates	CI	p	Estimates	CI	p
(Intercept)	0.01	-0.02 – 0.05	0.391	43.08	39.89 – 46.27	<0.001	82.60	80.85 – 84.34	<0.001	4.05	3.63 – 4.47	<0.001	4.11	3.86 – 4.36	<0.001	4.13	3.87 – 4.39	<0.001
Random Effects
σ2	0.15			317.10			282.66			2.79			2.07			2.17
τ00	0.00 type			4.61 type			0.10 type			0.17 type			0.06 type			0.06 type
ICC				0.01			0.00			0.06			0.03			0.03
N	4 type			4 type			4 type			4 type			4 type			4 type
Observations	480			216			372			960			960			960
Marginal R2 / Conditional R2	0.000 / NA			0.000 / 0.014			0.000 / 0.000			0.000 / 0.058			0.000 / 0.026			0.000 / 0.028

BF(icc_mod_rating_cl, hypothesis = "icc = 0; icc > 0")

First, unconstrained icc analysis...

Second, exploratory testing of icc's...
icc; 

Third, confirmatory testing of icc's...
icc=0; 
icc>0; 

Call:
BF.lmerMod(x = icc_mod_rating_cl, hypothesis = "icc = 0; icc > 0")

Bayesian hypothesis test
Type: confirmatory
Object: lmerMod
Parameter: intraclass correlations
Method: Bayes factors based on uniform priors

Posterior probabilities:
   Pr(hypothesis|data)
H1               0.375
H2               0.006
H3               0.619

Evidence matrix (Bayes factors):
      H1     H2    H3
H1 1.000 59.126 0.605
H2 0.017  1.000 0.010
H3 1.652 97.673 1.000

Hypotheses:
H1: icc=0
H2: icc>0
H3: complement

BF(icc_mod_rating_u3, hypothesis = "icc = 0; icc > 0")

First, unconstrained icc analysis...

Second, exploratory testing of icc's...
icc; 

Third, confirmatory testing of icc's...
icc=0; 
icc>0; 

Call:
BF.lmerMod(x = icc_mod_rating_u3, hypothesis = "icc = 0; icc > 0")

Bayesian hypothesis test
Type: confirmatory
Object: lmerMod
Parameter: intraclass correlations
Method: Bayes factors based on uniform priors

Posterior probabilities:
   Pr(hypothesis|data)
H1               0.657
H2               0.108
H3               0.235

Evidence matrix (Bayes factors):
      H1    H2    H3
H1 1.000 6.075 2.793
H2 0.165 1.000 0.460
H3 0.358 2.175 1.000

Hypotheses:
H1: icc=0
H2: icc>0
H3: complement

BF(icc_mod_rating_ov, hypothesis = "icc = 0; icc > 0")

First, unconstrained icc analysis...

Second, exploratory testing of icc's...
icc; 

Third, confirmatory testing of icc's...
icc=0; 
icc>0; 

Call:
BF.lmerMod(x = icc_mod_rating_ov, hypothesis = "icc = 0; icc > 0")

Bayesian hypothesis test
Type: confirmatory
Object: lmerMod
Parameter: intraclass correlations
Method: Bayes factors based on uniform priors

Posterior probabilities:
   Pr(hypothesis|data)
H1               0.605
H2               0.027
H3               0.368

Evidence matrix (Bayes factors):
      H1     H2    H3
H1 1.000 22.432 1.648
H2 0.045  1.000 0.073
H3 0.607 13.615 1.000

Hypotheses:
H1: icc=0
H2: icc>0
H3: complement

BF(icc_mod_diffi, hypothesis = "icc = 0; icc > 0")

First, unconstrained icc analysis...

Second, exploratory testing of icc's...
icc; 

Third, confirmatory testing of icc's...
icc=0; 
icc>0; 

Call:
BF.lmerMod(x = icc_mod_diffi, hypothesis = "icc = 0; icc > 0")

Bayesian hypothesis test
Type: confirmatory
Object: lmerMod
Parameter: intraclass correlations
Method: Bayes factors based on uniform priors

Posterior probabilities:
   Pr(hypothesis|data)
H1                   0
H2                   1
H3                   0

Evidence matrix (Bayes factors):
        H1 H2  H3
H1       1  0 Inf
H2 7895605  1 Inf
H3       0  0   1

Hypotheses:
H1: icc=0
H2: icc>0
H3: complement

BF(icc_mod_infor, hypothesis = "icc = 0; icc > 0")

First, unconstrained icc analysis...

Second, exploratory testing of icc's...
icc; 

Third, confirmatory testing of icc's...
icc=0; 
icc>0; 

Call:
BF.lmerMod(x = icc_mod_infor, hypothesis = "icc = 0; icc > 0")

Bayesian hypothesis test
Type: confirmatory
Object: lmerMod
Parameter: intraclass correlations
Method: Bayes factors based on uniform priors

Posterior probabilities:
   Pr(hypothesis|data)
H1               0.012
H2               0.988
H3               0.000

Evidence matrix (Bayes factors):
       H1    H2  H3
H1  1.000 0.012 Inf
H2 85.355 1.000 Inf
H3  0.000 0.000   1

Hypotheses:
H1: icc=0
H2: icc>0
H3: complement

BF(icc_mod_value, hypothesis = "icc = 0; icc > 0")

First, unconstrained icc analysis...

Second, exploratory testing of icc's...
icc; 

Third, confirmatory testing of icc's...
icc=0; 
icc>0; 

Call:
BF.lmerMod(x = icc_mod_value, hypothesis = "icc = 0; icc > 0")

Bayesian hypothesis test
Type: confirmatory
Object: lmerMod
Parameter: intraclass correlations
Method: Bayes factors based on uniform priors

Posterior probabilities:
   Pr(hypothesis|data)
H1               0.006
H2               0.994
H3               0.000

Evidence matrix (Bayes factors):
        H1    H2  H3
H1   1.000 0.006 Inf
H2 174.388 1.000 Inf
H3   0.000 0.000   1

Hypotheses:
H1: icc=0
H2: icc>0
H3: complement

Random Intercept Models With and Without Visualization Type

# Rating Cliffs Delta
mod0_rating_cl <- brm(rating_cl ~ + (1|session), 
                          data = study1_w,
                          iter = 20000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4)

Warning: Rows containing NAs were excluded from the model.

Compiling Stan program...

Trying to compile a simple C file

Start sampling

mod1_rating_cl <- brm(rating_cl ~ effsize + (1|session), 
                          data = study1_w,
                          iter = 20000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4)

Warning: Rows containing NAs were excluded from the model.

Compiling Stan program...

Trying to compile a simple C file

Start sampling

mod2_rating_cl <- brm(rating_cl ~ type + effsize + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )

Warning: Rows containing NAs were excluded from the model.

Compiling Stan program...

Trying to compile a simple C file

Start sampling

bayes_factor(mod1_rating_cl, mod0_rating_cl)

Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5

Estimated Bayes factor in favor of mod1_rating_cl over mod0_rating_cl: 693132216972923326062318468464640.00000

bayes_factor(mod2_rating_cl, mod1_rating_cl)

Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4

Estimated Bayes factor in favor of mod2_rating_cl over mod1_rating_cl: 0.00113

# Rating Cohen's U3
mod0_rating_u3 <- brm(rating_u3 ~ + (1|session), 
                          data = study1_w%>% 
                              filter(rating_u3_missconcept == F),
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )

Warning: Rows containing NAs were excluded from the model.

Compiling Stan program...

Trying to compile a simple C file

Start sampling

mod1_rating_u3 <- brm(rating_u3 ~ effsize + (1|session), 
                          data = study1_w%>% 
                              filter(rating_u3_missconcept == F),
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )

Warning: Rows containing NAs were excluded from the model.

Compiling Stan program...

Trying to compile a simple C file

Start sampling

mod2_rating_u3 <- brm(rating_u3 ~ type + effsize + (1|session), 
                          data = study1_w %>% 
                              filter(rating_u3_missconcept == F),
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )

Warning: Rows containing NAs were excluded from the model.

Compiling Stan program...

Trying to compile a simple C file

Start sampling

bayes_factor(mod1_rating_u3, mod0_rating_u3)

Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4

Estimated Bayes factor in favor of mod1_rating_u3 over mod0_rating_u3: 1078905234.49353

bayes_factor(mod2_rating_u3, mod1_rating_u3)

Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6

Estimated Bayes factor in favor of mod2_rating_u3 over mod1_rating_u3: 2459.95439

# Rating Overlap
mod0_rating_ov <- brm(rating_ov ~ + (1|session), 
                          data = study1_w%>% 
                              filter(rating_ov_missconcept == F),
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )

Warning: Rows containing NAs were excluded from the model.

Compiling Stan program...

Trying to compile a simple C file

Start sampling

mod1_rating_ov <- brm(rating_ov ~ effsize + (1|session), 
                          data = study1_w%>% 
                              filter(rating_ov_missconcept == F),
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )

Warning: Rows containing NAs were excluded from the model.

Compiling Stan program...

Trying to compile a simple C file

Start sampling

mod2_rating_ov <- brm(rating_ov ~ type + effsize + (1|session), 
                          data = study1_w %>% 
                              filter(rating_ov_missconcept == F),
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )

Warning: Rows containing NAs were excluded from the model.

Compiling Stan program...

Trying to compile a simple C file

Start sampling

bayes_factor(mod1_rating_ov, mod0_rating_ov)

Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6

Estimated Bayes factor in favor of mod1_rating_ov over mod0_rating_ov: 4.96682

bayes_factor(mod2_rating_ov, mod1_rating_ov)

Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5

Estimated Bayes factor in favor of mod2_rating_ov over mod1_rating_ov: 522.76670

# Perceived Difficulty
mod0_diffi <- brm(diffi ~ + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )

Compiling Stan program...

Trying to compile a simple C file

Start sampling

mod1_diffi <- brm(diffi ~ effsize + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )

Compiling Stan program...

Trying to compile a simple C file

Start sampling

mod2_diffi <- brm(diffi ~ type + effsize + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )

Compiling Stan program...

Trying to compile a simple C file

Start sampling

bayes_factor(mod1_diffi, mod0_diffi)

Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5

Estimated Bayes factor in favor of mod1_diffi over mod0_diffi: 0.39640

bayes_factor(mod2_diffi, mod1_diffi)

Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5

Estimated Bayes factor in favor of mod2_diffi over mod1_diffi: 4800845358254093.00000

# Perceived Informativity
mod0_infor <- brm(infor ~ + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )

Compiling Stan program...

Trying to compile a simple C file

Start sampling

mod1_infor <- brm(infor ~ effsize + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )

Compiling Stan program...

Trying to compile a simple C file

Start sampling

mod2_infor <- brm(infor ~ type + effsize + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )

Compiling Stan program...

Trying to compile a simple C file

Start sampling

bayes_factor(mod1_infor, mod0_infor)

Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5

Estimated Bayes factor in favor of mod1_infor over mod0_infor: 0.26039

bayes_factor(mod2_infor, mod1_infor)

Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4

Estimated Bayes factor in favor of mod2_infor over mod1_infor: 742781.83099

# Perceived Value 
mod0_value <- brm(value ~ + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )

Compiling Stan program...

Trying to compile a simple C file

Start sampling

mod1_value <- brm(value ~ effsize + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )

Compiling Stan program...

Trying to compile a simple C file

Start sampling

mod2_value <- brm(value ~ type + effsize + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )

Compiling Stan program...

Trying to compile a simple C file

Start sampling

bayes_factor(mod1_value, mod0_value)

Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5

Estimated Bayes factor in favor of mod1_value over mod0_value: 0.20633

bayes_factor(mod2_value, mod1_value)

Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6

Estimated Bayes factor in favor of mod2_value over mod1_value: 84181532.57805

sjPlot::tab_model(
    mod0_rating_cl, mod0_rating_u3, mod0_rating_ov,
    mod0_diffi, mod0_infor, mod0_value,
    mod1_rating_cl, mod1_rating_u3, mod1_rating_ov,
    mod1_diffi, mod1_infor, mod1_value,
    mod2_rating_cl, mod2_rating_u3, mod2_rating_ov,
    mod2_diffi, mod2_infor, mod2_value
    )

	rating_cl		rating_u3		rating_ov		diffi		infor		value		rating_cl		rating_u3		rating_ov		diffi		infor		value		rating_cl		rating_u3		rating_ov		diffi		infor		value
Predictors	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)
Intercept	0.01	-0.02 – 0.05	43.10	38.59 – 47.73	82.66	80.13 – 85.23	4.04	3.68 – 4.42	4.11	3.80 – 4.42	4.13	3.79 – 4.45	0.01	-0.02 – 0.04	43.32	38.67 – 48.08	82.68	80.23 – 85.21	4.04	3.68 – 4.41	4.11	3.81 – 4.41	4.12	3.78 – 4.47	0.02	-0.05 – 0.08	42.65	36.83 – 48.45	82.35	78.31 – 86.19	3.59	3.20 – 3.99	3.81	3.48 – 4.13	3.79	3.43 – 4.16
effsize													0.37	0.32 – 0.42	-11.82	-15.36 – -8.26	-1.03	-4.01 – 1.95	-0.09	-0.24 – 0.06	-0.06	-0.19 – 0.07	-0.05	-0.17 – 0.08	0.37	0.32 – 0.42	-11.64	-15.24 – -8.09	-1.07	-4.04 – 1.96	-0.09	-0.23 – 0.05	-0.06	-0.19 – 0.07	-0.05	-0.17 – 0.07
typehalfeye_xaxis																									-0.02	-0.10 – 0.06	0.58	-4.93 – 6.13	2.48	-2.19 – 7.14	0.65	0.42 – 0.87	0.39	0.19 – 0.59	0.51	0.32 – 0.70
typehalfeye_yaxis																									0.01	-0.08 – 0.09	4.15	-1.60 – 9.78	-1.01	-5.84 – 3.84	0.97	0.74 – 1.19	0.60	0.40 – 0.80	0.58	0.39 – 0.77
typeraincloud_yaxis																									0.01	-0.08 – 0.09	-1.87	-7.60 – 3.81	-0.14	-4.90 – 4.67	0.24	0.02 – 0.47	0.24	0.04 – 0.45	0.23	0.04 – 0.42
Random Effects
σ2	0.15		259.01		254.58		1.72		1.32		1.17		0.11		213.63		255.25		1.72		1.32		1.17		0.11		212.62		256.13		1.58		1.27		1.11
τ00	0.00 session		75.49 session		30.02 session		1.30 session		0.86 session		1.15 session		0.00 session		82.97 session		29.99 session		1.31 session		0.87 session		1.15 session		0.00 session		80.38 session		28.66 session		1.32 session		0.87 session		1.13 session
ICC	0.01		0.23		0.11		0.43		0.39		0.50		0.02		0.28		0.11		0.43		0.40		0.50		0.02		0.27		0.10		0.46		0.40		0.50
N	40 session		18 session		31 session		40 session		40 session		40 session		40 session		18 session		31 session		40 session		40 session		40 session		40 session		18 session		31 session		40 session		40 session		40 session
Observations	480		216		372		960		960		960		480		216		372		960		960		960		480		216		372		960		960		960
Marginal R2 / Conditional R2	0.000 / 0.006		0.000 / 0.198		0.000 / 0.099		0.000 / 0.413		0.000 / 0.376		0.000 / 0.476		0.278 / 0.291		0.134 / 0.344		0.002 / 0.102		0.001 / 0.415		0.001 / 0.377		0.000 / 0.476		0.282 / 0.294		0.154 / 0.358		0.015 / 0.111		0.050 / 0.464		0.025 / 0.401		0.026 / 0.501

Perceived Difficulty

Graphichal Overview

study1_w %>% 
    ggplot(aes(type, diffi)) +
    geom_jitter() +
    stat_summary(fun.data = mean_sdl,
                 fun.args = list(mult = 1),
                 color = "white") +
    theme_modern_rc() +
    labs(title = "Difficulty",
         subtitle = "per Plot Type",
         caption = "Means ± 1*SD")

Perceived Informativity

Graphical Overview

study1_w %>% 
    ggplot(aes(type, infor)) +
    geom_jitter() +
    stat_summary(fun.data = mean_sdl,
                 fun.args = list(mult = 1),
                 color = "white") +
    theme_modern_rc() +
    labs(title = "Informativity",
         subtitle = "per Plot Type",
         caption = "Means ± 1*SD")

Perceived Value

Graphical Overview

study1_w %>% 
    ggplot(aes(type, value)) +
    geom_jitter() +
    stat_summary(fun.data = mean_sdl,
                 fun.args = list(mult = 1),
                 color = "white") +
    theme_modern_rc() +
    labs(title = "Perceived Value",
         subtitle = "per Plot Type",
         caption = "Means ± 1*SD")

Sensitivity

ggplot(study1_w %>% 
           select(sensi_ordinal, type) %>% 
           na.omit(), aes(sensi_ordinal, 
                          color = sensi_ordinal,
                          fill = sensi_ordinal)) +
    facet_wrap(~ type) +
    geom_bar() +
    theme_modern_rc() +
    theme(strip.text = element_text(color = "white")) +
    ggtitle("Sensitivity", "all participants")

ggplot(study1_w %>% 
           select(sensi_ordinal, type, session) %>% 
           na.omit() %>% 
           filter(!session %in% ov_misconceptualizers & 
                      !session %in% u3_misconceptualizers), aes(sensi_ordinal, 
                          color = sensi_ordinal,
                          fill = sensi_ordinal)) +
    facet_wrap(~ type) +
    geom_bar() +
    theme_modern_rc() +
    theme(strip.text = element_text(color = "white")) +
    ggtitle("Sensitivity", "without missconceptualizers")

ggplot(study1_w %>% 
           select(sensi_binary_filtered, type, session),
       aes(sensi_binary_filtered, 
           color = sensi_binary_filtered,
           fill = sensi_binary_filtered)) +
    facet_wrap(~ type) +
    geom_bar() +
    theme_modern_rc() +
    theme(strip.text = element_text(color = "white")) +
    ggtitle("Binary Sensitivity", "without wrong decisions")

Fitting a series of logistic regressions with non-informative priors

logreg_mod0 <- brm(
    sensi_binary_filtered ~ +(1 | session),
    family = bernoulli(link = "logit"),
    data = study1_w,
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4
    
)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

logreg_mod1 <-
    brm(
        sensi_binary_filtered ~ effsize_abs + (1 | session),
        family = bernoulli(link = "logit"),
        data = study1_w,
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4
        
    )

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

logreg_mod2 <-
    brm(
        sensi_binary_filtered ~ effsize_abs + type + (1 | session),
        family = bernoulli(link = "logit"),
        data = study1_w,
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4
        
    )

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

bayes_factor(logreg_mod1, logreg_mod0)

Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6

Estimated Bayes factor in favor of logreg_mod1 over logreg_mod0: 979646073651857.50000

bayes_factor(logreg_mod2, logreg_mod1)

Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5

Estimated Bayes factor in favor of logreg_mod2 over logreg_mod1: 2.14972

sjPlot::tab_model(logreg_mod0, logreg_mod1, logreg_mod2, show.icc = TRUE)

	sensi_binary_filtered		sensi_binary_filtered		sensi_binary_filtered
Predictors	Odds Ratios	CI (95%)	Odds Ratios	CI (95%)	Odds Ratios	CI (95%)
Intercept	7.75	5.20 – 12.94	1.67	0.96 – 3.10	1.45	0.72 – 2.93
effsize_abs			36.73	14.94 – 91.86	37.60	15.08 – 99.41
typehalfeye_xaxis					1.56	0.89 – 2.84
typehalfeye_yaxis					1.36	0.78 – 2.42
typeraincloud_yaxis					0.95	0.55 – 1.63
Random Effects
σ2	3.29		3.29		3.29
τ00	1.32 session		1.61 session		1.65 session
ICC	0.29		0.33		0.33
N	40 session		40 session		40 session
Observations	933		933		933
Marginal R2 / Conditional R2	0.000 / 0.115		0.051 / 0.200		0.056 / 0.208

pp_check(logreg_mod2) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

Using 10 posterior draws for ppc type 'dens_overlay' by default.

Scale for colour is already present.
Adding another scale for colour, which will replace the existing scale.

Efficiency

Visualisation

Raw data

ggplot(study1_w_timestamp, aes(as.factor(plotNrWithin), effic)) +
    geom_boxplot(alpha = .2, color = "lightgrey") +
    geom_sina(alpha = .5) +
    coord_cartesian(ylim = c(0,100000)) + 
    facet_wrap(~type) +
    theme_modern_rc() +
    labs(title ="Dwell Times Until First Decision",
          subtitle = "Per Plot Type and Plot Repetition") +
    theme(strip.text = element_text(color = "white"))

5% Percent Truncated

ggplot(study1_w_timestamp, aes(as.factor(plotNrWithin), effic_05righttrunc)) +
    geom_boxplot(alpha = .2, color = "lightgrey") +
    geom_sina(alpha = .5) +
    coord_cartesian(ylim = c(0,85000)) + 
    facet_wrap(~type) +
    theme_modern_rc() +
    labs(title ="5% Truncated Dwell Times Until First Decision",
          subtitle = "Per Plot Type and Plot Repetition") +
    theme(strip.text = element_text(color = "white"))

Warning: Removed 48 rows containing non-finite values (`stat_boxplot()`).

Warning: Removed 48 rows containing non-finite values (`stat_sina()`).

10% Percent Truncated

ggplot(study1_w_timestamp, aes(as.factor(plotNrWithin), effic_10righttrunc)) +
    geom_boxplot(alpha = .2, color = "lightgrey") +
    geom_sina(alpha = .5) +
    coord_cartesian(ylim = c(0,85000)) + 
    facet_wrap(~type) +
    theme_modern_rc() +
    labs(title ="10% Truncated Dwell Times Until First Decision",
          subtitle = "Per Plot Type and Plot Repetition") +
    theme(strip.text = element_text(color = "white"))

Warning: Removed 96 rows containing non-finite values (`stat_boxplot()`).

Warning: Removed 96 rows containing non-finite values (`stat_sina()`).

log() Transformed

ggplot(study1_w_timestamp, 
       aes(as.factor(plotNrWithin), 
           log(effic_05righttrunc))) +
    geom_boxplot(alpha = .2, color = "lightgrey") +
    geom_sina(alpha = .5) +
    facet_wrap(~type) +
    theme_modern_rc() +
    labs(title ="log Transformed Dwell Times Until First Decision",
          subtitle = "Per Plot Type and Plot Repetition") +
    theme(strip.text = element_text(color = "white"))

Warning: Removed 48 rows containing non-finite values (`stat_boxplot()`).

Warning: Removed 48 rows containing non-finite values (`stat_sina()`).

Modeling with {brms}

Efficiency for First Plot

study1_w_timestamp_effsize <- study1_w_timestamp %>% 
    full_join(study1_w %>% select(session, effsize), by = "session")

plot01_mod0 <-
    brm(log_effic_05righttrunc ~ 1 + (1 | session),
        data = study1_w_timestamp_effsize %>%
            filter(plotNrWithin == 1),
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4,
        iter = 6000)

pp_check(plot01_mod0) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

plot01_mod1 <-
    brm(log_effic_05righttrunc~ 1 + effsize + (1 | session),
        data = study1_w_timestamp_effsize %>%
            filter(plotNrWithin == 1),
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4,
        iter = 6000)

pp_check(plot01_mod1) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

plot01_mod2 <-
    brm(log_effic_05righttrunc ~ 1 + effsize + type + (1 | session),
        data = study1_w_timestamp_effsize %>%
            filter(plotNrWithin == 1),
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4,
        iter = 6000)

pp_check(plot01_mod2) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

sjPlot::tab_model(plot01_mod0, plot01_mod1, plot01_mod2)
bayes_factor(plot01_mod1, plot01_mod0)
bayes_factor(plot01_mod2, plot01_mod1)

Efficiency for Last Three Plots

plot0306_mod0 <-
    brm(log_effic_05righttrunc ~ 1 + plotNrWithin_factor + (1 | session),
        data = study1_w_timestamp_effsize %>%
            filter(plotNrWithin >= 4),
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4,
        iter = 6000)

pp_check(plot0306_mod0) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

plot0306_mod1 <-
    brm(log_effic_05righttrunc ~ 1 + plotNrWithin_factor + effsize +
            (1 | session),
        data = study1_w_timestamp_effsize %>%
            filter(plotNrWithin >= 4),
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4)

pp_check(plot0306_mod1) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

plot0306_mod2 <-
    brm(log_effic_05righttrunc ~ 1 + plotNrWithin_factor + effsize +
            type + (1 | session),
        data = study1_w_timestamp_effsize %>%
            filter(plotNrWithin >= 4),
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4)

pp_check(plot0306_mod2) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

sjPlot::tab_model(plot0306_mod0, plot0306_mod1, plot0306_mod2)

	log_effic_05righttrunc		log_effic_05righttrunc		log_effic_05righttrunc
Predictors	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)
Intercept	9.48	9.34 – 9.62	9.48	9.35 – 9.60	9.58	9.45 – 9.71
plotNrWithin_factor: plotNrWithin_factor5	-0.01	-0.03 – 0.02	-0.01	-0.03 – 0.02	-0.01	-0.03 – 0.01
plotNrWithin_factor: plotNrWithin_factor6	-0.19	-0.21 – -0.17	-0.19	-0.21 – -0.17	-0.19	-0.21 – -0.17
effsize			-0.00	-0.02 – 0.02	0.00	-0.02 – 0.02
typehalfeye_xaxis					-0.13	-0.16 – -0.11
typehalfeye_yaxis					-0.21	-0.23 – -0.18
typeraincloud_yaxis					-0.06	-0.09 – -0.04
Random Effects
σ2	0.24		0.24		0.24
τ00	0.17 session		0.17 session		0.16 session
ICC	0.41		0.41		0.41
N	40 session		40 session		40 session
Observations	10944		10944		10944
Marginal R2 / Conditional R2	0.019 / 0.406		0.019 / 0.406		0.034 / 0.421

bayes_factor(plot0306_mod1, plot0306_mod0)

Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Iteration: 7
Iteration: 8
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Iteration: 7

Estimated Bayes factor in favor of plot0306_mod1 over plot0306_mod0: 0.02010

bayes_factor(plot0306_mod2, plot0306_mod1)

Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Iteration: 7
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6

Estimated Bayes factor in favor of plot0306_mod2 over plot0306_mod1: 18618875449292113148499011020035359633188591249536319488.00000

Research Question 2

Contrast Effects

Graphical Overview

# preparing multilevel model with brms for graphical overview

# rating u3
mod_plot_ratingu3 <- brm(
    scale(rating_u3) ~ (1 | type),
    data = study1_w,
    control = list(adapt_delta = .999),
    cores = 4,
    iter = 20000
    
) 

Warning: Rows containing NAs were excluded from the model.

Compiling Stan program...

Trying to compile a simple C file

Start sampling

Warning: There were 4 divergent transitions after warmup. See
https://mc-stan.org/misc/warnings.html#divergent-transitions-after-warmup
to find out why this is a problem and how to eliminate them.

Warning: Examine the pairs() plot to diagnose sampling problems

plot_ratingu3 <- mod_plot_ratingu3 %>%
  spread_draws(b_Intercept, r_type[type,]) %>%
 # median_qi(type_mean = b_Intercept + r_type, .width = c(.95, .66)) %>%
  mutate(type_mean = b_Intercept + r_type) %>%
  ggplot(aes(y = type, x = type_mean
             #xmin = .lower, xmax = .upper
             )) +
#  geom_pointinterval() +
    stat_halfeye() +
    theme_ipsum_rc() +
    theme(axis.title.x = element_blank(),
          axis.text.y = element_text(hjust = 0)) +
    #scale_x_continuous(limits=c(-0.6, 0.6)) +
    ggtitle('Accuracy in U3 Metrics')

Warning: `gather_()` was deprecated in tidyr 1.2.0.
ℹ Please use `gather()` instead.
ℹ The deprecated feature was likely used in the tidybayes package.
  Please report the issue at <]8;;https://github.com/mjskay/tidybayes/issues/newhttps://github.com/mjskay/tidybayes/issues/new]8;;>.

plot_ratingu3

Warning: Using the `size` aesthietic with geom_segment was deprecated in ggplot2 3.4.0.
ℹ Please use the `linewidth` aesthetic instead.

# rating ov
mod_plot_ratingov <- brm (
    scale(rating_ov) ~ (1 | type),
    data = study1_w,
    control = list(adapt_delta = .999),
    cores = 4,
    iter = 10000
    
)

Warning: Rows containing NAs were excluded from the model.

Compiling Stan program...

Trying to compile a simple C file

Start sampling

plot_ratingov <-mod_plot_ratingov %>%
  spread_draws(b_Intercept, r_type[type,]) %>%
 # median_qi(type_mean = b_Intercept + r_type, .width = c(.95, .66)) %>%
  mutate(type_mean = b_Intercept + r_type) %>%
  ggplot(aes(y = type, x = type_mean
             #xmin = .lower, xmax = .upper
             )) +
#  geom_pointinterval() +
    stat_halfeye() + 
    theme_ipsum_rc()+
    theme(axis.title.x = element_blank(),
          axis.text.y = element_blank(), 
          axis.title.y = element_blank()) +
    #scale_x_continuous(limits=c(-0.6, 0.6)) +
    ggtitle('Accuracy in Overlap Metrics')

plot_ratingov

# difficulty
mod_plot_diffi <- brm (
    scale(diffi) ~ (1 | type),
    data = study1_w,
    control = list(adapt_delta = .999),
    cores = 4
    
)

Compiling Stan program...

Trying to compile a simple C file

Start sampling

plot_diffi <- mod_plot_diffi %>%
  spread_draws(b_Intercept, r_type[type,]) %>%
 # median_qi(type_mean = b_Intercept + r_type, .width = c(.95, .66)) %>%
  mutate(type_mean = b_Intercept + r_type) %>%
  ggplot(aes(y = type, x = type_mean
             #xmin = .lower, xmax = .upper
             )) +
#  geom_pointinterval() +
    stat_halfeye() +
    theme_ipsum_rc() +
    theme(axis.title.x = element_blank(),
          axis.text.y = element_blank(), 
          axis.title.y = element_blank()) +
    #scale_x_continuous(limits=c(-0.6, 0.6)) +
    ggtitle('Perceived Task Difficulty')

plot_diffi

# informativity
mod_plot_infor <- brm (
    scale(infor) ~ (1 | type),
    data = study1_w,
    control = list(adapt_delta = .999),
    cores = 4,
    iter = 10000
    
)

Compiling Stan program...

Trying to compile a simple C file

Start sampling

plot_infor <- mod_plot_infor %>%
  spread_draws(b_Intercept, r_type[type,]) %>%
 # median_qi(type_mean = b_Intercept + r_type, .width = c(.95, .66)) %>%
  mutate(type_mean = b_Intercept + r_type) %>%
  ggplot(aes(y = type, x = type_mean #xmin = .lower, xmax = .upper
             )) +
#  geom_pointinterval() +
  stat_halfeye() +
    theme_ipsum_rc() +
    theme(axis.title.x = element_blank(),
          axis.text.y = element_text(hjust = 0)) +
    #scale_x_continuous(limits=c(-0.6, 0.6)) +
    ggtitle('Perceived Informativity')

plot_infor

# value 

mod_plot_value <- brm (
    scale(value) ~ (1 | type),
    data = study1_w,
    control = list(adapt_delta = .999),
    cores = 4
    
)

Compiling Stan program...

Trying to compile a simple C file

Start sampling

plot_value <- mod_plot_value %>%
  spread_draws(b_Intercept, r_type[type,]) %>%
 # median_qi(type_mean = b_Intercept + r_type, .width = c(.95, .66)) %>%
  mutate(type_mean = b_Intercept + r_type) %>%
  ggplot(aes(y = type, x = type_mean #xmin = .lower, xmax = .upper
             )) +
#  geom_pointinterval() +
  stat_halfeye() +
    theme_ipsum_rc() +
    theme(axis.title.x = element_blank(),
          axis.text.y = element_blank(), 
          axis.title.y = element_blank()) +
    #scale_x_continuous(limits=c(-0.6, 0.6)) +
    ggtitle('Perceived Value')

plot_value

# efficiency for the first plot

mod_plot_efficiency1 <-
    brm(
        scale(effic) ~ (1 | type),
        data = study1_w_timestamp_effsize %>%
            filter(plotNrWithin == 1),
        control = list(adapt_delta = .999),
        cores = 4
        
    )

Compiling Stan program...

Trying to compile a simple C file

Start sampling

Warning: There were 3 transitions after warmup that exceeded the maximum treedepth. Increase max_treedepth above 10. See
https://mc-stan.org/misc/warnings.html#maximum-treedepth-exceeded

Warning: Examine the pairs() plot to diagnose sampling problems

plot_efficiency1 <- mod_plot_efficiency1 %>%
  spread_draws(b_Intercept, r_type[type,]) %>%
 # median_qi(type_mean = b_Intercept + r_type, .width = c(.95, .66)) %>%
  mutate(type_mean = b_Intercept + r_type) %>%
  ggplot(aes(y = type, x = type_mean #xmin = .lower, xmax = .upper
             )) +
#  geom_pointinterval() +
  stat_halfeye() +
    theme_ipsum_rc() +
    theme(axis.title.x = element_blank(),
          axis.text.y = element_blank(), 
          axis.title.y = element_blank()) +
    #scale_x_continuous(limits=c(-0.6, 0.6)) +
    ggtitle('Efficiency for the First Plot')

plot_efficiency1

plot_contrasteffects <- plot_ratingu3 + plot_ratingov + plot_diffi + plot_infor + plot_value + plot_efficiency1 + 
    plot_layout(ncol=3)

ggsave("plot_contrasteffects.png", 
       plot_contrasteffects,
       width = 25*.55,
       height = 16*.55)

Model Contrast Effects with {brms}

# create dummy variables to compare visualization types for those measures that showed evidence for difference (accuracy U3, accuracy overlap, perceived task difficulty, perceived informativity, perceived value, efficiency)

## Gardner Altman Plot vs. Halfeye xaxis
gardneraltman_vs_halfeye_x <- study1_w %>%
    group_by(session) %>%
    mutate(dummy_gardneraltman_halfeye_x = 
               case_when(type == "gardneraltman_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "gardneraltman_xaxis" | type == "halfeye_xaxis") %>%
    ungroup()

gardneraltman_vs_halfeye_x_effic <- study1_w_timestamp %>%
    group_by(session) %>%
    mutate(dummy_gardneraltman_halfeye_x = 
               case_when(type == "gardneraltman_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "gardneraltman_xaxis" | type == "halfeye_xaxis") %>%
    ungroup() %>% 
    full_join(study1_w %>% select(session, effsize), by = "session")

mod_accu3_gardneraltman_halfeye_x <- 
    brm(scale(rating_u3) ~ dummy_gardneraltman_halfeye_x + effsize + (1|session), 
                          data = gardneraltman_vs_halfeye_x%>% 
                              filter(rating_u3_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

mod_accov_gardneraltman_halfeye_x <- 
    brm(scale(rating_ov) ~ dummy_gardneraltman_halfeye_x + effsize + (1|session), 
                          data = gardneraltman_vs_halfeye_x%>% 
                              filter(rating_ov_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

mod_diffi_gardneraltman_halfeye_x <- 
    brm(scale(diffi) ~ dummy_gardneraltman_halfeye_x + effsize 
        + (1|session), 
                          data = gardneraltman_vs_halfeye_x,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)

Trying to compile a simple C file

mod_infor_gardneraltman_halfeye_x <- 
    brm(scale(infor) ~ dummy_gardneraltman_halfeye_x + effsize
        + (1|session), 
                          data = gardneraltman_vs_halfeye_x,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)

Trying to compile a simple C file

mod_value_gardneraltman_halfeye_x <- 
    brm(scale(value) ~ dummy_gardneraltman_halfeye_x + effsize 
        + (1|session), 
                          data = gardneraltman_vs_halfeye_x,
                          save_pars = save_pars(all = TRUE))

Compiling Stan program...

Trying to compile a simple C file

Start sampling

Warning: Bulk Effective Samples Size (ESS) is too low, indicating posterior means and medians may be unreliable.
Running the chains for more iterations may help. See
https://mc-stan.org/misc/warnings.html#bulk-ess

mod_effic01_gardneraltman_halfeye_x <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_gardneraltman_halfeye_x + 
            effsize + (1|session),
    data = gardneraltman_vs_halfeye_x_effic %>%  
    filter(plotNrWithin == 1),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

mod_effic0306_gardneraltman_halfeye_x <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_gardneraltman_halfeye_x +
            effsize + (1|session),
    data = gardneraltman_vs_halfeye_x_effic %>% 
        filter(plotNrWithin >= 4),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

## Gardner Altman Plot vs. Halfeye yaxis
gardneraltman_vs_halfeye_y <- study1_w %>%
    group_by(session) %>%
    mutate(dummy_gardneraltman_halfeye_y = 
               case_when(type == "gardneraltman_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "gardneraltman_xaxis" | type == "halfeye_yaxis") %>%
    ungroup()

gardneraltman_vs_halfeye_y_effic <- study1_w_timestamp %>%
    group_by(session) %>%
    mutate(dummy_gardneraltman_halfeye_y = 
               case_when(type == "gardneraltman_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "gardneraltman_xaxis" | type == "halfeye_yaxis") %>%
    ungroup()  %>% 
    full_join(study1_w %>% select(session, effsize), by = "session")


mod_accu3_gardneraltman_halfeye_y <- 
    brm(scale(rating_u3) ~ dummy_gardneraltman_halfeye_y + effsize + (1|session), 
                          data = gardneraltman_vs_halfeye_y%>% 
                              filter(rating_u3_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

mod_accov_gardneraltman_halfeye_y <- 
    brm(scale(rating_ov) ~ dummy_gardneraltman_halfeye_y + effsize + (1|session), 
                          data = gardneraltman_vs_halfeye_y%>% 
                              filter(rating_ov_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

mod_diffi_gardneraltman_halfeye_y <- 
    brm(scale(diffi) ~ dummy_gardneraltman_halfeye_y + effsize 
        + (1|session), 
                          data = gardneraltman_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

Trying to compile a simple C file

mod_infor_gardneraltman_halfeye_y <- 
    brm(scale(infor) ~ dummy_gardneraltman_halfeye_y + effsize 
        + (1|session), 
                          data = gardneraltman_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

Trying to compile a simple C file

mod_value_gardneraltman_halfeye_y <- 
    brm(scale(value) ~ dummy_gardneraltman_halfeye_y + effsize 
        + (1|session), 
                          data = gardneraltman_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

Trying to compile a simple C file

mod_effic01_gardneraltman_halfeye_y <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_gardneraltman_halfeye_y +
            effsize + (1|session),
    data = gardneraltman_vs_halfeye_y_effic %>%  
    filter(plotNrWithin == 1),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

Warning: Bulk Effective Samples Size (ESS) is too low, indicating posterior means and medians may be unreliable.
Running the chains for more iterations may help. See
https://mc-stan.org/misc/warnings.html#bulk-ess

Warning: Tail Effective Samples Size (ESS) is too low, indicating posterior variances and tail quantiles may be unreliable.
Running the chains for more iterations may help. See
https://mc-stan.org/misc/warnings.html#tail-ess

mod_effic0306_gardneraltman_halfeye_y <- 
    brm(log_effic_05righttrunc ~ dummy_gardneraltman_halfeye_y + 
            effsize + (1|session),
    data = gardneraltman_vs_halfeye_y_effic %>% 
        filter(plotNrWithin >= 4),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

## Raincloud Plot vs. Halfeye x axis
raincloud_vs_halfeye_x <- study1_w %>%
    group_by(session) %>%
    mutate(dummy_raincloud_halfeye_x = 
               case_when(type == "raincloud_yaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "halfeye_xaxis" | type == "raincloud_yaxis") %>%
    ungroup()

raincloud_vs_halfeye_x_effic <- study1_w_timestamp %>%
    group_by(session) %>%
    mutate(dummy_raincloud_halfeye_x = 
               case_when(type == "raincloud_yaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "raincloud_yaxis" | type == "halfeye_xaxis") %>%
    ungroup() %>% 
    full_join(study1_w %>% select(session, effsize), by = "session")

mod_accu3_raincloud_halfeye_x <- 
    brm(scale(rating_u3) ~ dummy_raincloud_halfeye_x + effsize + (1|session), 
                          data = raincloud_vs_halfeye_x %>% 
                              filter(rating_u3_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

mod_accov_raincloud_halfeye_x <- 
    brm(scale(rating_ov) ~ dummy_raincloud_halfeye_x + effsize + (1|session), 
                          data = raincloud_vs_halfeye_x %>% 
                              filter(rating_ov_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

mod_diffi_raincloud_halfeye_x <- 
    brm(scale(diffi) ~ dummy_raincloud_halfeye_x + 
            effsize + (1|session), 
                          data = raincloud_vs_halfeye_x,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

Trying to compile a simple C file

mod_infor_raincloud_halfeye_x <- 
    brm(scale(infor) ~ dummy_raincloud_halfeye_x +
        effsize + (1|session), 
                          data = raincloud_vs_halfeye_x,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

Trying to compile a simple C file

mod_value_raincloud_halfeye_x <- 
    brm(scale(value) ~ dummy_raincloud_halfeye_x + 
            effsize + (1|session), 
                          data = raincloud_vs_halfeye_x,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

Trying to compile a simple C file

mod_effic01_raincloud_halfeye_x <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_raincloud_halfeye_x + 
            effsize + (1|session),
    data = raincloud_vs_halfeye_x_effic %>%  
    filter(plotNrWithin == 1),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

mod_effic0306_raincloud_halfeye_x <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_raincloud_halfeye_x +
            effsize + (1|session),
    data = raincloud_vs_halfeye_x_effic %>% 
        filter(plotNrWithin >= 4),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

##  Raincloud Plot vs.Halfeye yaxis
raincloud_vs_halfeye_y <- study1_w %>%
    group_by(session) %>%
    mutate(dummy_raincloud_halfeye_y = 
               case_when(type == "raincloud_yaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "halfeye_yaxis" | type == "raincloud_yaxis") %>%
    ungroup()

raincloud_vs_halfeye_y_effic <- study1_w_timestamp %>%
    group_by(session) %>%
    mutate(dummy_raincloud_halfeye_y = 
               case_when(type == "raincloud_yaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "raincloud_yaxis" | type == "halfeye_yaxis") %>%
    ungroup() %>% 
    full_join(study1_w %>% select(session, effsize), by = "session")

mod_accu3_raincloud_halfeye_y <- 
    brm(scale(rating_u3) ~ dummy_raincloud_halfeye_y + effsize + (1|session), 
                          data = raincloud_vs_halfeye_y %>% 
                              filter(rating_u3_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

mod_accov_raincloud_halfeye_y <- 
    brm(scale(rating_ov) ~ dummy_raincloud_halfeye_y + effsize + (1|session), 
                          data = raincloud_vs_halfeye_y %>% 
                              filter(rating_ov_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

mod_diffi_raincloud_halfeye_y <- 
    brm(scale(diffi) ~ dummy_raincloud_halfeye_y + 
            effsize + (1|session), 
                          data = raincloud_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

Trying to compile a simple C file

mod_infor_raincloud_halfeye_y <- 
    brm(scale(infor) ~ dummy_raincloud_halfeye_y + 
            effsize + (1|session), 
                          data = raincloud_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

Trying to compile a simple C file

mod_value_raincloud_halfeye_y <- 
    brm(scale(value) ~ dummy_raincloud_halfeye_y + 
            effsize + (1|session), 
                          data = raincloud_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

Trying to compile a simple C file

mod_effic01_raincloud_halfeye_y <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_raincloud_halfeye_y +
            effsize + (1|session),
    data = raincloud_vs_halfeye_y_effic %>%  
    filter(plotNrWithin == 1),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

mod_effic0306_raincloud_halfeye_y <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_raincloud_halfeye_y +
            effsize + (1|session),
    data = raincloud_vs_halfeye_y_effic %>% 
        filter(plotNrWithin >= 4),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

sjPlot::tab_model(mod_accu3_gardneraltman_halfeye_x, 
        mod_accov_gardneraltman_halfeye_x, mod_diffi_gardneraltman_halfeye_x, 
        mod_infor_gardneraltman_halfeye_x, mod_value_gardneraltman_halfeye_x,
        mod_effic01_gardneraltman_halfeye_x, mod_effic0306_gardneraltman_halfeye_x,
        mod_accu3_gardneraltman_halfeye_y, mod_accov_gardneraltman_halfeye_y, 
        mod_diffi_gardneraltman_halfeye_y, mod_infor_gardneraltman_halfeye_y, 
        mod_value_gardneraltman_halfeye_y,
        mod_effic01_gardneraltman_halfeye_y, mod_effic0306_gardneraltman_halfeye_y,
        mod_accov_raincloud_halfeye_x, mod_diffi_raincloud_halfeye_x, 
        mod_infor_raincloud_halfeye_x, mod_value_raincloud_halfeye_x,
        mod_effic01_raincloud_halfeye_x, mod_effic0306_raincloud_halfeye_x,
        mod_accu3_raincloud_halfeye_y, mod_accov_raincloud_halfeye_y, 
        mod_diffi_raincloud_halfeye_y, mod_infor_raincloud_halfeye_y, 
        mod_value_raincloud_halfeye_y,
        mod_effic01_raincloud_halfeye_y, mod_effic0306_raincloud_halfeye_y)

	scale(rating_u3)		scale(rating_ov)		scale(diffi)		scale(infor)		scale(value)		scale(log_effic_05righttrunc)		scale(log_effic_05righttrunc)		scale(rating_u3)		scale(rating_ov)		scale(diffi)		scale(infor)		scale(value)		scale(log_effic_05righttrunc)		log_effic_05righttrunc		scale(rating_ov)		scale(diffi)		scale(infor)		scale(value)		scale(log_effic_05righttrunc)		scale(log_effic_05righttrunc)		scale(rating_u3)		scale(rating_ov)		scale(diffi)		scale(infor)		scale(value)		scale(log_effic_05righttrunc)		scale(log_effic_05righttrunc)
Predictors	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)
Intercept	-0.01	-0.35 – 0.33	-0.11	-0.35 – 0.13	-0.19	-0.44 – 0.08	-0.13	-0.38 – 0.12	-0.19	-0.43 – 0.06	0.34	0.08 – 0.58	0.10	-0.12 – 0.30	-0.16	-0.50 – 0.16	0.01	-0.26 – 0.27	-0.26	-0.51 – -0.03	-0.21	-0.42 – 0.02	-0.20	-0.45 – 0.05	0.42	0.16 – 0.67	9.51	9.39 – 9.64	-0.11	-0.34 – 0.11	-0.12	-0.37 – 0.11	-0.05	-0.27 – 0.18	-0.09	-0.32 – 0.16	-0.04	-0.31 – 0.25	0.07	-0.16 – 0.29	-0.16	-0.46 – 0.18	0.04	-0.20 – 0.27	-0.22	-0.46 – 0.01	-0.13	-0.34 – 0.10	-0.12	-0.36 – 0.12	0.05	-0.24 – 0.33	0.14	-0.09 – 0.38
dummy_gardneraltman_halfeye_x	0.04	-0.27 – 0.36	0.18	-0.11 – 0.46	0.37	0.26 – 0.49	0.26	0.14 – 0.39	0.35	0.23 – 0.46	-0.69	-0.75 – -0.64	-0.20	-0.24 – -0.16
effsize	-0.73	-1.00 – -0.45	0.05	-0.22 – 0.31	-0.04	-0.15 – 0.06	-0.02	-0.13 – 0.09	-0.05	-0.16 – 0.05	0.00	-0.05 – 0.05	-0.00	-0.04 – 0.04	-0.50	-0.82 – -0.18	-0.08	-0.32 – 0.18	-0.04	-0.14 – 0.07	-0.03	-0.15 – 0.09	-0.01	-0.13 – 0.10	0.00	-0.04 – 0.04	0.00	-0.02 – 0.02	-0.07	-0.33 – 0.17	-0.07	-0.19 – 0.05	-0.06	-0.19 – 0.07	-0.05	-0.16 – 0.06	-0.00	-0.05 – 0.05	0.00	-0.04 – 0.04	-0.54	-0.84 – -0.22	-0.15	-0.41 – 0.10	-0.07	-0.18 – 0.05	-0.06	-0.19 – 0.06	-0.01	-0.13 – 0.11	0.00	-0.04 – 0.04	0.00	-0.03 – 0.03
dummy_gardneraltman_halfeye_y															0.29	-0.06 – 0.64	-0.04	-0.32 – 0.24	0.54	0.41 – 0.66	0.40	0.27 – 0.54	0.39	0.27 – 0.52	-0.79	-0.83 – -0.74	-0.21	-0.23 – -0.18
dummy_raincloud_halfeye_x																													0.21	-0.08 – 0.50	0.25	0.12 – 0.39	0.10	-0.03 – 0.24	0.19	0.07 – 0.31	0.15	0.10 – 0.20	-0.12	-0.16 – -0.08
dummy_raincloud_halfeye_y																																									0.35	-0.02 – 0.73	-0.05	-0.34 – 0.23	0.43	0.30 – 0.57	0.26	0.12 – 0.39	0.24	0.11 – 0.36	-0.00	-0.05 – 0.05	-0.23	-0.27 – -0.19
Random Effects
σ2	0.64		0.95		0.43		0.46		0.40		0.35		0.58		0.78		0.83		0.46		0.55		0.47		0.25		0.23		0.98		0.55		0.62		0.50		0.31		0.54		0.78		0.92		0.54		0.60		0.54		0.26		0.51
τ00	0.27 session		0.07 session		0.59 session		0.57 session		0.62 session		0.60 session		0.43 session		0.20 session		0.22 session		0.51 session		0.45 session		0.53 session		0.69 session		0.17 session		0.04 session		0.48 session		0.41 session		0.54 session		0.79 session		0.50 session		0.15 session		0.10 session		0.46 session		0.42 session		0.49 session		0.87 session		0.53 session
ICC	0.29		0.07		0.58		0.55		0.61		0.63		0.43		0.20		0.21		0.52		0.45		0.53		0.73		0.42		0.04		0.47		0.40		0.52		0.72		0.48		0.16		0.10		0.46		0.41		0.48		0.77		0.51
N	18 session		31 session		40 session		40 session		40 session		39 session		40 session		18 session		31 session		40 session		40 session		40 session		40 session		40 session		31 session		40 session		40 session		40 session		39 session		40 session		18 session		31 session		40 session		40 session		40 session		40 session		40 session
Observations	109		185		480		480		480		1824		5472		104		177		480		480		480		1824		5472		195		480		480		480		1824		5472		107		187		480		480		480		1824		5472
Marginal R2 / Conditional R2	0.169 / 0.393		0.014 / 0.081		0.036 / 0.579		0.019 / 0.543		0.032 / 0.608		0.118 / 0.654		0.010 / 0.418		0.107 / 0.267		0.008 / 0.203		0.074 / 0.540		0.042 / 0.452		0.039 / 0.533		0.151 / 0.752		0.027 / 0.419		0.018 / 0.055		0.019 / 0.460		0.005 / 0.385		0.011 / 0.508		0.006 / 0.688		0.004 / 0.461		0.132 / 0.264		0.013 / 0.113		0.050 / 0.468		0.019 / 0.407		0.015 / 0.471		0.000 / 0.744		0.013 / 0.488

## Gardner Altman Plot vs. Raincloud Plot
gardneraltman_vs_raincloud <- study1_w %>%
    group_by(session) %>%
    mutate(dummy_gardneraltman_raincloud = 
               case_when(type == "gardneraltman_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "gardneraltman_xaxis" | type == "raincloud_yaxis") %>%
    ungroup()

gardneraltman_vs_raincloud_effic <- study1_w_timestamp %>%
    group_by(session) %>%
    mutate(dummy_gardneraltman_raincloud = 
               case_when(type == "gardneraltman_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "gardneraltman_xaxis" | type == "raincloud_yaxis") %>%
    ungroup() %>% 
    full_join(study1_w %>% select(session, effsize), by = "session")

mod_accu3_gardneraltman_raincloud <- 
    brm(scale(rating_u3) ~ dummy_gardneraltman_raincloud + effsize + (1|session), 
                          data = gardneraltman_vs_raincloud%>% 
                              filter(rating_u3_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

mod_accov_gardneraltman_raincloud <- 
    brm(scale(rating_ov) ~ dummy_gardneraltman_raincloud + effsize + (1|session), 
                          data = gardneraltman_vs_raincloud%>% 
                              filter(rating_ov_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

mod_diffi_gardneraltman_raincloud <- 
    brm(scale(diffi) ~ dummy_gardneraltman_raincloud + 
            effsize + (1|session), 
                          data = gardneraltman_vs_raincloud,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

Trying to compile a simple C file

mod_infor_gardneraltman_raincloud <- 
    brm(scale(infor) ~ dummy_gardneraltman_raincloud +
            effsize + (1|session), 
        data = gardneraltman_vs_raincloud,
        save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

Trying to compile a simple C file

mod_value_gardneraltman_raincloud <- 
    brm(scale(value) ~ dummy_gardneraltman_raincloud + 
            effsize + (1|session), 
                          data = gardneraltman_vs_raincloud,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

Trying to compile a simple C file

mod_effic01_gardneraltman_raincloud <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_gardneraltman_raincloud + 
            effsize + (1|session),
    data = gardneraltman_vs_raincloud_effic %>%  
    filter(plotNrWithin == 1),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

mod_effic0306_gardneraltman_raincloud <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_gardneraltman_raincloud + 
            effsize + (1|session),
    data = gardneraltman_vs_raincloud_effic %>% 
        filter(plotNrWithin >= 4),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

## Halfeye xaxis vs. Halfeye yaxis

halfeye_x_vs_halfeye_y <- study1_w %>%
    group_by(session) %>%
    mutate(dummy_halfeye_x_halfeye_y = 
               case_when(type == "halfeye_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "halfeye_xaxis" | type == "halfeye_yaxis") %>%
    ungroup()

halfeye_x_vs_halfeye_y_effic <- study1_w_timestamp %>%
    group_by(session) %>%
    mutate(dummy_halfeye_x_halfeye_y = 
               case_when(type == "halfeye_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "halfeye_xaxis" | type == "halfeye_yaxis") %>%
    ungroup() %>% 
    full_join(study1_w %>% select(session, effsize), by = "session")

mod_accu3_halfeye_x_halfeye_y <- 
    brm(scale(rating_u3) ~ dummy_halfeye_x_halfeye_y + 
            effsize + (1|session), 
                          data = halfeye_x_vs_halfeye_y %>% 
                              filter(rating_u3_missconcept == F),
                          save_pars = save_pars(all = TRUE),
        
    silent = 2,
    refresh = 0,
    cores = 4)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

mod_accov_halfeye_x_halfeye_y <- 
    brm(scale(rating_ov) ~ dummy_halfeye_x_halfeye_y + 
            effsize + (1|session), 
                          data = halfeye_x_vs_halfeye_y%>% 
                              filter(rating_ov_missconcept == F),
                          save_pars = save_pars(all = TRUE),
        
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 20000)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

Warning: There were 1 divergent transitions after warmup. See
https://mc-stan.org/misc/warnings.html#divergent-transitions-after-warmup
to find out why this is a problem and how to eliminate them.

Warning: Examine the pairs() plot to diagnose sampling problems

mod_diffi_halfeye_x_halfeye_y <- 
    brm(scale(diffi) ~ dummy_halfeye_x_halfeye_y +
            effsize + (1|session), 
                          data = halfeye_x_vs_halfeye_y,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE))

Compiling Stan program...

Trying to compile a simple C file

Start sampling

mod_infor_halfeye_x_halfeye_y <- 
    brm(scale(infor) ~ dummy_halfeye_x_halfeye_y + 
            effsize + (1|session), 
                          data = halfeye_x_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

Trying to compile a simple C file

mod_value_halfeye_x_halfeye_y <- 
    brm(scale(value) ~ dummy_halfeye_x_halfeye_y + 
            effsize + (1|session), 
                          data = halfeye_x_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)

Trying to compile a simple C file

mod_effic01_halfeye_x_halfeye_y <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_halfeye_x_halfeye_y + 
            effsize + (1|session),
    data = halfeye_x_vs_halfeye_y_effic %>%  
    filter(plotNrWithin == 1),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    iter = 10000)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

mod_effic0306_halfeye_x_halfeye_y <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_halfeye_x_halfeye_y + 
            effsize + (1|session),
    data = halfeye_x_vs_halfeye_y_effic%>% 
        filter(plotNrWithin >= 4),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

sjPlot::tab_model(mod_accu3_halfeye_x_halfeye_y, mod_accov_halfeye_x_halfeye_y, 
                  mod_diffi_halfeye_x_halfeye_y, mod_infor_halfeye_x_halfeye_y, 
                  mod_value_halfeye_x_halfeye_y,
                  mod_effic01_halfeye_x_halfeye_y)

	scale(rating_u3)		scale(rating_ov)		scale(diffi)		scale(infor)		scale(value)		scale(log_effic_05righttrunc)
Predictors	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)
Intercept	-0.10	-0.42 – 0.23	0.11	-0.10 – 0.32	-0.10	-0.34 – 0.16	-0.09	-0.33 – 0.15	-0.03	-0.29 – 0.24	0.10	-0.16 – 0.38
dummy_halfeye_x_halfeye_y	0.17	-0.15 – 0.49	-0.23	-0.51 – 0.05	0.20	0.08 – 0.33	0.16	0.03 – 0.29	0.05	-0.06 – 0.16	-0.14	-0.20 – -0.08
effsize	-0.67	-0.98 – -0.38	-0.04	-0.31 – 0.22	-0.01	-0.12 – 0.10	0.04	-0.07 – 0.14	0.05	-0.05 – 0.16	-0.00	-0.05 – 0.05
Random Effects
σ2	0.70		0.98		0.48		0.47		0.41		0.37
τ00	0.25 session		0.03 session		0.56 session		0.57 session		0.65 session		0.72 session
ICC	0.26		0.03		0.53		0.55		0.62		0.66
N	18 session		31 session		40 session		40 session		40 session		39 session
Observations	109		194		480		480		480		1824
Marginal R2 / Conditional R2	0.150 / 0.337		0.019 / 0.052		0.011 / 0.521		0.008 / 0.537		0.002 / 0.598		0.005 / 0.626

sjPlot::tab_model(mod_effic0306_halfeye_x_halfeye_y,
                  mod_accu3_gardneraltman_raincloud, 
                  mod_accov_gardneraltman_raincloud, 
                  mod_diffi_gardneraltman_raincloud)

	scale(log_effic_05righttrunc)		scale(rating_u3)		scale(rating_ov)		scale(diffi)
Predictors	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)
Intercept	0.08	-0.15 – 0.31	0.12	-0.21 – 0.45	0.02	-0.23 – 0.27	-0.07	-0.31 – 0.16
dummy_halfeye_x_halfeye_y	-0.12	-0.16 – -0.09
effsize	0.00	-0.03 – 0.03	-0.69	-0.97 – -0.39	-0.10	-0.36 – 0.15	-0.09	-0.21 – 0.03
dummy_gardneraltman_raincloud			-0.17	-0.49 – 0.15	-0.01	-0.30 – 0.27	0.14	0.01 – 0.27
Random Effects
σ2	0.52		0.68		0.89		0.59
τ00	0.53 session		0.24 session		0.15 session		0.45 session
ICC	0.51		0.26		0.14		0.44
N	40 session		18 session		31 session		40 session
Observations	5472		107		178		480
Marginal R2 / Conditional R2	0.004 / 0.482		0.161 / 0.354		0.010 / 0.148		0.009 / 0.420

sjPlot::tab_model(mod_infor_gardneraltman_raincloud, 
                  mod_value_gardneraltman_raincloud,
                  mod_effic01_gardneraltman_raincloud, 
                  mod_effic0306_gardneraltman_raincloud)

	scale(infor)		scale(value)		scale(log_effic_05righttrunc)		scale(log_effic_05righttrunc)
Predictors	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)	Estimates	CI (95%)
Intercept	-0.08	-0.30 – 0.15	-0.08	-0.33 – 0.16	0.41	0.15 – 0.68	0.05	-0.17 – 0.27
dummy_gardneraltman_raincloud	0.16	0.02 – 0.29	0.15	0.02 – 0.27	-0.77	-0.81 – -0.72	-0.10	-0.14 – -0.06
effsize	-0.11	-0.23 – 0.01	-0.11	-0.23 – -0.00	-0.00	-0.04 – 0.04	-0.00	-0.04 – 0.04
Random Effects
σ2	0.60		0.50		0.25		0.57
τ00	0.43 session		0.54 session		0.70 session		0.45 session
ICC	0.42		0.52		0.73		0.44
N	40 session		40 session		39 session		40 session
Observations	480		480		1824		5472
Marginal R2 / Conditional R2	0.011 / 0.406		0.010 / 0.506		0.144 / 0.747		0.003 / 0.426

References

Arslan, Ruben C., Matthias P. Walther, and Cyril S. Tata. 2020. “Formr: A Study Framework Allowing for Automated Feedback Generation and Complex Longitudinal Experience-Sampling Studies Using R.” Behavior Research Methods 52 (1): 376–87. https://doi.org/10.3758/s13428-019-01236-y.